!nvidia-smi

Mon May 31 16:59:13 2021       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 465.19.01    Driver Version: 460.32.03    CUDA Version: 11.2     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |
| N/A   51C    P0    31W / 250W |      0MiB / 16280MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+
                                                                               
+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|  No running processes found                                                 |
+-----------------------------------------------------------------------------+

!pip install git+https://github.com/ChristophReich1996/Involution

Collecting git+https://github.com/ChristophReich1996/Involution
  Cloning https://github.com/ChristophReich1996/Involution to /tmp/pip-req-build-o5j0akgz
  Running command git clone -q https://github.com/ChristophReich1996/Involution /tmp/pip-req-build-o5j0akgz
Requirement already satisfied: torch>=1.7.0 in /usr/local/lib/python3.7/dist-packages (from involution==0.2) (1.8.1+cu101)
Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=1.7.0->involution==0.2) (3.7.4.3)
Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torch>=1.7.0->involution==0.2) (1.19.5)
Building wheels for collected packages: involution
  Building wheel for involution (setup.py) ... done
  Created wheel for involution: filename=involution-0.2-cp37-none-any.whl size=4603 sha256=da2b422827c14502322225e46b584e4e5a3c1f61cc6f7596bc75656a47bb6ccc
  Stored in directory: /tmp/pip-ephem-wheel-cache-9r_6iymf/wheels/d2/b3/9b/ef9398d783f0609e8ddca4cd465f7840d074741ba5c4f7fae4
Successfully built involution
Installing collected packages: involution
Successfully installed involution-0.2

# Import necessary libraries

import torch
import torchvision
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, random_split, SubsetRandomSampler
from torchvision.datasets import CIFAR10
from torchvision import datasets, transforms
from torch.optim import *

import os
import random
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import math
import cv2
import glob
import copy

from sklearn.model_selection import train_test_split
#import wandb

from torchsummary import summary

from skimage.feature import hog
from tqdm import tqdm as tqdm

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

seed = 42

from numba import jit, cuda

import torch
import numpy as np
from torch.utils.data import TensorDataset, DataLoader, Dataset

from PIL import Image
import glob

import albumentations as A
from albumentations.pytorch import ToTensor


import torch
from involution import Involution2d

#involution = Involution2d(in_channels=32, out_channels=64)
#output = involution(torch.rand(1, 32, 128, 128))

# Mount google drive to colab

from google.colab import drive
drive.mount('/content/drive', force_remount=True)

Mounted at /content/drive

!ls -lrt /content/drive/MyDrive/CV_Project/Video

total 16687
-rw------- 1 root root  6890854 May 31 17:01 Foggy.mp4
-rw------- 1 root root 10196325 May 31 18:10 Foggy_Forest.mov

!ls -lrt

total 8
drwxr-xr-x 1 root root 4096 May  6 13:44 sample_data
drwx------ 6 root root 4096 May 31 18:12 drive

class Block_en(nn.Module):
    def __init__(self, in_ch, out_ch):
        super().__init__()
        self.conv1 = nn.Conv2d(in_ch, out_ch, 3, padding = 1)
        self.inonv1 = Involution2d(in_channels=in_ch, out_channels=out_ch, kernel_size = (3,3), padding = (1,1))
        self.relu  = nn.ReLU()
        self.conv2 = nn.Conv2d(out_ch, out_ch, 3, padding = 1)
        self.inonv2 = Involution2d(in_channels=out_ch, out_channels=out_ch, kernel_size = (3,3), padding = (1,1))
    
    def forward(self, x):
        #return self.relu(self.conv2(self.relu(self.conv1(x))))
        #print(self.inonv1(x).shape)
        return self.relu(self.conv2(self.relu(self.inonv1(x))))
        #return self.relu(self.inonv2(self.relu(self.inonv1(x))))

class Block_de(nn.Module):
    def __init__(self, in_ch, out_ch):
        super().__init__()
        self.conv1 = nn.Conv2d(in_ch, out_ch, 3, padding = 1)
        #self.inonv1 = Involution2d(in_channels=in_ch, out_channels=out_ch, kernel_size = (3,3), padding = (1,1))
        self.relu  = nn.ReLU()
        self.conv2 = nn.Conv2d(out_ch, out_ch, 3, padding = 1)
        #self.inonv2 = Involution2d(in_channels=out_ch, out_channels=out_ch, kernel_size = (3,3), padding = (1,1))
    
    def forward(self, x):
        return self.relu(self.conv2(self.relu(self.conv1(x))))
        #print(self.inonv1(x).shape)
        #return self.relu(self.conv2(self.relu(self.inonv1(x))))

class Encoder(nn.Module):
    def __init__(self, chs=(3,64,128,256,512,1024)):
        super().__init__()
        self.enc_blocks = nn.ModuleList([Block_en(chs[i], chs[i+1]) for i in range(len(chs)-1)])
        self.pool       = nn.MaxPool2d(2)
    
    def forward(self, x):
        ftrs = []
        for block in self.enc_blocks:
            x = block(x)
            ftrs.append(x)
            x = self.pool(x)
        return ftrs

class Decoder(nn.Module):
    def __init__(self, chs=(1024, 512, 256, 128, 64)):
        super().__init__()
        self.chs         = chs
        self.upconvs    = nn.ModuleList([nn.ConvTranspose2d(chs[i], chs[i+1], 2, 2) for i in range(len(chs)-1)])
        self.dec_blocks = nn.ModuleList([Block_de(chs[i], chs[i+1]) for i in range(len(chs)-1)]) 
        
    def forward(self, x, encoder_features):
        for i in range(len(self.chs)-1):
            x        = self.upconvs[i](x)
            enc_ftrs = self.crop(encoder_features[i], x)
            x        = torch.cat([x, enc_ftrs], dim=1)
            x        = self.dec_blocks[i](x)
        return x
    
    def crop(self, enc_ftrs, x):
        _, _, H, W = x.shape
        enc_ftrs   = torchvision.transforms.CenterCrop([H, W])(enc_ftrs)
        return enc_ftrs

class InvolutionUNet(nn.Module):
    def __init__(self, enc_chs=(3,64,128,256,512), dec_chs=(512, 256, 128, 64), num_class=1, retain_dim=False, out_sz=(572,572)):
        super().__init__()
        self.encoder     = Encoder(enc_chs)
        self.decoder     = Decoder(dec_chs)
        self.head        = nn.Conv2d(dec_chs[-1], num_class, 1)
        self.retain_dim  = retain_dim
        self.out_sz = out_sz

    def forward(self, x):
        enc_ftrs = self.encoder(x)
        out      = self.decoder(enc_ftrs[::-1][0], enc_ftrs[::-1][1:])
        out      = self.head(out)
        if self.retain_dim:
            out = F.interpolate(out, self.out_sz)
        return out

Inference on the Best Model

test_transform = A.Compose(
    [
        #A.CenterCrop(height=224, width=224),
        A.Resize(height=720, width=720),
        A.Normalize(mean=(0.64, 0.6, 0.58),std=(0.14,0.15, 0.152)),
        ToTensor(),
    ])

best_model = InvolutionUNet(enc_chs=(3,64,128,256), dec_chs=(256, 128, 64), num_class=3, retain_dim=False, out_sz=(128,128))

best_ckp = torch.load('/content/drive/MyDrive/CV_Project/CheckPoints/r3/unet-14.pt')

best_model.load_state_dict(best_ckp['model'])

<All keys matched successfully>

def dehaze_video(video, model, transform, video_path = 'DeHazed.avi'):
  fps = video.get(cv2.CAP_PROP_FPS)
  out = cv2.VideoWriter(video_path,cv2.VideoWriter_fourcc('M','J','P','G'), fps, (720,720))
  frame_count = 0
  while(video.isOpened()):
    ret_val, new_frame = video.read()
    #print(new_frame.shape)
    mask = np.zeros_like(new_frame)
    frame_count = frame_count + 1
    if frame_count > video.get(cv2.CAP_PROP_FRAME_COUNT):
      return video_path
    if ret_val is True:
      new_frame = cv2.cvtColor(new_frame, cv2.COLOR_BGR2RGB)
      transformed = transform(image=new_frame, mask=mask)
      new_frame_transformed = transformed['image']
      #plt.imshow(new_frame_transformed.permute(1,2,0))
      #break
      model.eval()
      with torch.no_grad():
        images_fod = torch.tensor(new_frame_transformed).to(device).unsqueeze(0)
        model = model.to(device)
        pred_masks_fod = model(images_fod)
        #maxValue = np.amax(pred_masks_fod.detach().cpu().numpy())
        #minValue = np.amin(pred_masks_fod.detach().cpu().numpy())
        pred_masks_fod = (np.clip(pred_masks_fod.detach().cpu().squeeze(0).permute(1,2,0).numpy(), 0 , 1)*255).astype('uint8')
      '''
      if frame_count == 10:
        #print((pred_masks_fod.detach().cpu().squeeze(0).permute(1,2,0).numpy()))
        #plt.imshow(np.round_(pred_masks_fod.detach().cpu().squeeze(0).permute(1,2,0).numpy()))
        #plt.imshow((pred_masks_fod.detach().cpu().squeeze(0).permute(1,2,0).numpy()*255).astype('uint8'))
        print(maxValue)
        print(minValue)
        print("***********")
    
        print(pred_masks_fod)
        plt.imshow(pred_masks_fod)
        break
      '''
      #out.write((pred_masks_fod.detach().cpu().squeeze(0).permute(1,2,0).numpy()*255).astype('uint8'))
      out.write(pred_masks_fod)

video_path = '/content/drive/MyDrive/CV_Project/Video/Foggy_Forest.mov'
video = cv2.VideoCapture(video_path)

dehaze_video(video, best_model, test_transform, video_path = 'Defogged_Forest.avi')

/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:20: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).

'Defogged_Forest.avi'

!ffmpeg -i Defogged_Forest.avi Defogged_Forest.mp4

ffmpeg version 3.4.8-0ubuntu0.2 Copyright (c) 2000-2020 the FFmpeg developers
  built with gcc 7 (Ubuntu 7.5.0-3ubuntu1~18.04)
  configuration: --prefix=/usr --extra-version=0ubuntu0.2 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared
  libavutil      55. 78.100 / 55. 78.100
  libavcodec     57.107.100 / 57.107.100
  libavformat    57. 83.100 / 57. 83.100
  libavdevice    57. 10.100 / 57. 10.100
  libavfilter     6.107.100 /  6.107.100
  libavresample   3.  7.  0 /  3.  7.  0
  libswscale      4.  8.100 /  4.  8.100
  libswresample   2.  9.100 /  2.  9.100
  libpostproc    54.  7.100 / 54.  7.100
Input #0, avi, from 'Defogged_Forest.avi':
  Metadata:
    encoder         : Lavf58.35.100
  Duration: 00:00:29.83, start: 0.000000, bitrate: 20152 kb/s
    Stream #0:0: Video: mjpeg (MJPG / 0x47504A4D), yuvj420p(pc, bt470bg/unknown/unknown), 720x720, 20173 kb/s, 24.97 fps, 24.97 tbr, 24.97 tbn, 24.97 tbc
Stream mapping:
  Stream #0:0 -> #0:0 (mjpeg (native) -> h264 (libx264))
Press [q] to stop, [?] for help
[libx264 @ 0x5600ef349e00] using cpu capabilities: MMX2 SSE2Fast SSSE3 SSE4.2 AVX FMA3 BMI2 AVX2
[libx264 @ 0x5600ef349e00] profile High, level 3.1
[libx264 @ 0x5600ef349e00] 264 - core 152 r2854 e9a5903 - H.264/MPEG-4 AVC codec - Copyleft 2003-2017 - http://www.videolan.org/x264.html - options: cabac=1 ref=3 deblock=1:0:0 analyse=0x3:0x113 me=hex subme=7 psy=1 psy_rd=1.00:0.00 mixed_ref=1 me_range=16 chroma_me=1 trellis=1 8x8dct=1 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=-2 threads=6 lookahead_threads=1 sliced_threads=0 nr=0 decimate=1 interlaced=0 bluray_compat=0 constrained_intra=0 bframes=3 b_pyramid=2 b_adapt=1 b_bias=0 direct=1 weightb=1 open_gop=0 weightp=2 keyint=250 keyint_min=24 scenecut=40 intra_refresh=0 rc_lookahead=40 rc=crf mbtree=1 crf=23.0 qcomp=0.60 qpmin=0 qpmax=69 qpstep=4 ip_ratio=1.40 aq=1:1.00
Output #0, mp4, to 'Defogged_Forest.mp4':
  Metadata:
    encoder         : Lavf57.83.100
    Stream #0:0: Video: h264 (libx264) (avc1 / 0x31637661), yuvj420p(pc), 720x720, q=-1--1, 24.97 fps, 24973 tbn, 24.97 tbc
    Metadata:
      encoder         : Lavc57.107.100 libx264
    Side data:
      cpb: bitrate max/min/avg: 0/0/0 buffer size: 0 vbv_delay: -1
frame=  745 fps= 44 q=-1.0 Lsize=    9696kB time=00:00:29.71 bitrate=2673.2kbits/s speed=1.77x    
video:9686kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.099915%
[libx264 @ 0x5600ef349e00] frame I:3     Avg QP:22.92  size: 78158
[libx264 @ 0x5600ef349e00] frame P:187   Avg QP:24.91  size: 36996
[libx264 @ 0x5600ef349e00] frame B:555   Avg QP:28.67  size:  4982
[libx264 @ 0x5600ef349e00] consecutive B-frames:  0.7%  0.0%  0.0% 99.3%
[libx264 @ 0x5600ef349e00] mb I  I16..4:  0.7% 95.0%  4.3%
[libx264 @ 0x5600ef349e00] mb P  I16..4:  0.5% 11.9%  1.2%  P16..4: 27.0% 34.1% 22.8%  0.0%  0.0%    skip: 2.5%
[libx264 @ 0x5600ef349e00] mb B  I16..4:  0.1%  0.7%  0.0%  B16..8: 43.0%  8.7%  2.0%  direct: 4.4%  skip:41.0%  L0:28.0% L1:39.9% BI:32.2%
[libx264 @ 0x5600ef349e00] 8x8 transform intra:87.6% inter:65.9%
[libx264 @ 0x5600ef349e00] coded y,uvDC,uvAC intra: 76.0% 71.8% 37.0% inter: 28.2% 20.0% 2.0%
[libx264 @ 0x5600ef349e00] i16 v,h,dc,p: 72% 14% 12%  2%
[libx264 @ 0x5600ef349e00] i8 v,h,dc,ddl,ddr,vr,hd,vl,hu: 26% 10% 41%  3%  4%  5%  3%  4%  4%
[libx264 @ 0x5600ef349e00] i4 v,h,dc,ddl,ddr,vr,hd,vl,hu: 53%  7% 13%  4%  4%  7%  3%  6%  3%
[libx264 @ 0x5600ef349e00] i8c dc,h,v,p: 49% 15% 30%  6%
[libx264 @ 0x5600ef349e00] Weighted P-Frames: Y:1.6% UV:0.0%
[libx264 @ 0x5600ef349e00] ref P L0: 57.3% 20.3% 20.0%  2.4%  0.0%
[libx264 @ 0x5600ef349e00] ref B L0: 94.7%  4.6%  0.6%
[libx264 @ 0x5600ef349e00] ref B L1: 98.5%  1.5%
[libx264 @ 0x5600ef349e00] kb/s:2659.61

!cp ./Defogged_Forest.mp4 /content/drive/MyDrive/CV_Project/Video_Output/

Foggy Video

from IPython.display import HTML
from base64 import b64encode
mp4 = open('/content/drive/MyDrive/CV_Project/Video/Foggy_Forest.mov','rb').read()
data_url = "data:video/mp4;base64," + b64encode(mp4).decode()

HTML("""
<video controls>
      <source src="%s" type="video/mp4">
</video>
""" % data_url)

Dehazed Video

from IPython.display import HTML
from base64 import b64encode
mp4 = open('/content/drive/MyDrive/CV_Project/Video_Output/Defogged_Forest.mp4','rb').read()
data_url = "data:video/mp4;base64," + b64encode(mp4).decode()

HTML("""
<video controls>
      <source src="%s" type="video/mp4">
</video>
""" % data_url)